Model Backtesting

Code
import eia_backtesting
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
import os
import datetime
import requests
import json
Support for Torch based models not available. To enable them, install "darts", "u8darts[torch]" or "u8darts[all]" (with pip); or "u8darts-torch" or "u8darts-all" (with conda).
/opt/ai_dev_workshop/lib/python3.10/site-packages/statsforecast/core.py:27: TqdmExperimentalWarning:

Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)

Load data

Code
raw_json = open("../settings/series.json")
meta_json = json.load(raw_json)

meta_path = meta_json["meta_path"]
data_path = meta_json["data_path"]
leaderboard_path = meta_json["leaderboard_path"]

input = pd.read_csv(data_path)

input["period"] = pd.to_datetime(input["period"])

end = input.groupby(['subba'])['period'].max().min().floor(freq = "d") - datetime.timedelta(hours = 1)

print(end)


input.head()
2024-06-14 23:00:00
period subba subba-name parent parent-name value value-units
0 2018-07-01 08:00:00 PGAE Pacific Gas and Electric CISO California Independent System Operator 12522.0 megawatthours
1 2018-07-01 09:00:00 PGAE Pacific Gas and Electric CISO California Independent System Operator 11745.0 megawatthours
2 2018-07-01 10:00:00 PGAE Pacific Gas and Electric CISO California Independent System Operator 11200.0 megawatthours
3 2018-07-01 11:00:00 PGAE Pacific Gas and Electric CISO California Independent System Operator 10822.0 megawatthours
4 2018-07-01 12:00:00 PGAE Pacific Gas and Electric CISO California Independent System Operator 10644.0 megawatthours
Code
p = px.line(input, x="period", y="value", color="subba",
 labels={
                     "period": "Time",
                     "value": "MegaWattHours",
                     "subba": "Sub-Region"
                 },
title = "California Hourly Demand By Operating Provider")
p.show()
Code
freq = meta_json["backtesting"]["freq"]
h = meta_json["backtesting"]["h"]
overlap = meta_json["backtesting"]["overlap"]
tags = meta_json["backtesting"]["tags"]
experiment_name = meta_json["backtesting"]["experiment_name"]
mlflow_path = meta_json["backtesting"]["mlflow_path"]
p = meta_json["backtesting"]["p"]
pi = meta_json["backtesting"]["pi"]
quantiles = meta_json["backtesting"]["quantiles"]
seed = meta_json["backtesting"]["seed"]


params1 = meta_json["backtesting"]["models"]["model1"]
params2 = meta_json["backtesting"]["models"]["model2"]
params3 = meta_json["backtesting"]["models"]["model3"]
params4 = meta_json["backtesting"]["models"]["model4"]
params5 = meta_json["backtesting"]["models"]["model5"]
params6 = meta_json["backtesting"]["models"]["model6"]
params7 = meta_json["backtesting"]["models"]["model7"]

params = [params1, params2, params3, params4, params5, params7]

for i in range(len(params)):
    params[i]["h"] = h
    params[i]["freq"] = freq
    params[i]["quantiles"] = quantiles
    params[i]["pi"] = pi
    params[i]["seed"] = seed
Code
series = pd.DataFrame(meta_json["series"])

backtesting_output = {}
for index, row in series.iterrows():
    print("Start backtesting for " + row['subba_id'])
    df = None
    df = input[input["subba"] == row['subba_id']]
    df = df.sort_values(by = ["period"])
    par = params
    for i in range(len(par)):
        par[i]["subba"] = row['subba_id']

    backtesting_output[row['subba_id']] = eia_backtesting.backtesting(input = df, 
            partitions=p, 
            overlap = overlap, 
            h = h, 
            params = par,
            experiment_name = experiment_name + "_" + row['subba_id'],
            mlflow_path = mlflow_path,
            overwrite = True,
            tags = tags)
Code
leaderboard = None
best = None

for index, row in series.iterrows():
    subba = row["subba_id"]
    print(subba)
    print(backtesting_output[subba].leaderboard)
    leader_subba = backtesting_output[subba].leaderboard
    leader_subba["subba"] = subba

    if leaderboard is None:
        leaderboard = backtesting_output[subba].leaderboard
        best =  leader_subba[leader_subba["mape"] == leader_subba["mape"].min()]
    else:
        leaderboard_temp = backtesting_output[subba].leaderboard
        leaderboard = leaderboard._append(leaderboard_temp)
        best = best._append(leader_subba[leader_subba["mape"] == leader_subba["mape"].min()])

    
#best = best[["subba", "model","model_label" ,"mape", "rmse"]] 
best.to_csv(leaderboard_path, index = False)   

best
PGAE
  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.089334  1365.539586   
1     2024-06-14      model2  LinearRegressionModel  0.090086  1374.423744   
2     2024-06-14      model3  LinearRegressionModel  0.096589  1456.745612   
3     2024-06-14      model4  LinearRegressionModel  0.091571  1415.877250   
4     2024-06-14      model5  LinearRegressionModel  0.092669  1404.666182   
5     2024-06-14      model7               XGBModel  0.091234  1333.922406   

   coverage                                           comments  
0  0.725000  LM model with lags, training with 2 years of h...  
1  0.729167  LM model with lags, training with 3 years of h...  
2  0.537500                                 Model 2 with lag 1  
3  0.725000                       Model 1 with additional lags  
4  0.725000                       Model 1 with additional lags  
5  0.670833                                  XGBoost with lags  
SCE
  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.052259   824.558770   
1     2024-06-14      model2  LinearRegressionModel  0.051878   817.262735   
2     2024-06-14      model3  LinearRegressionModel  0.077653  1139.373683   
3     2024-06-14      model4  LinearRegressionModel  0.068839  1073.844087   
4     2024-06-14      model5  LinearRegressionModel  0.071172  1121.572839   
5     2024-06-14      model7               XGBModel  0.077308  1153.257120   

   coverage                                           comments  
0  0.929167  LM model with lags, training with 2 years of h...  
1  0.929167  LM model with lags, training with 3 years of h...  
2  0.695833                                 Model 2 with lag 1  
3  0.887500                       Model 1 with additional lags  
4  0.862500                       Model 1 with additional lags  
5  0.775000                                  XGBoost with lags  
SDGE
  forecast_label model_label                  model      mape        rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.083502  187.714005   
1     2024-06-14      model2  LinearRegressionModel  0.083141  188.657150   
2     2024-06-14      model3  LinearRegressionModel  0.131489  266.710639   
3     2024-06-14      model4  LinearRegressionModel  0.125430  257.151173   
4     2024-06-14      model5  LinearRegressionModel  0.123442  251.715517   
5     2024-06-14      model7               XGBModel  0.129947  261.224281   

   coverage                                           comments  
0  0.933333  LM model with lags, training with 2 years of h...  
1  0.933333  LM model with lags, training with 3 years of h...  
2  0.770833                                 Model 2 with lag 1  
3  0.879167                       Model 1 with additional lags  
4  0.887500                       Model 1 with additional lags  
5  0.875000                                  XGBoost with lags  
VEA
  forecast_label model_label                  model      mape       rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.100615  18.513913   
1     2024-06-14      model2  LinearRegressionModel  0.100654  18.651437   
2     2024-06-14      model3  LinearRegressionModel  0.150168  30.266325   
3     2024-06-14      model4  LinearRegressionModel  0.127715  25.427010   
4     2024-06-14      model5  LinearRegressionModel  0.129275  25.454507   
5     2024-06-14      model7               XGBModel  0.114773  22.695727   

   coverage                                           comments  
0  0.887500  LM model with lags, training with 2 years of h...  
1  0.833333  LM model with lags, training with 3 years of h...  
2  0.787500                                 Model 2 with lag 1  
3  0.850000                       Model 1 with additional lags  
4  0.850000                       Model 1 with additional lags  
5  0.858333                                  XGBoost with lags  
forecast_label model_label model mape rmse coverage comments subba
0 2024-06-14 model1 LinearRegressionModel 0.089334 1365.539586 0.725000 LM model with lags, training with 2 years of h... PGAE
1 2024-06-14 model2 LinearRegressionModel 0.051878 817.262735 0.929167 LM model with lags, training with 3 years of h... SCE
1 2024-06-14 model2 LinearRegressionModel 0.083141 188.657150 0.933333 LM model with lags, training with 3 years of h... SDGE
0 2024-06-14 model1 LinearRegressionModel 0.100615 18.513913 0.887500 LM model with lags, training with 2 years of h... VEA
Code
print(leaderboard)
  forecast_label model_label                  model      mape         rmse  \
0     2024-06-14      model1  LinearRegressionModel  0.089334  1365.539586   
1     2024-06-14      model2  LinearRegressionModel  0.090086  1374.423744   
2     2024-06-14      model3  LinearRegressionModel  0.096589  1456.745612   
3     2024-06-14      model4  LinearRegressionModel  0.091571  1415.877250   
4     2024-06-14      model5  LinearRegressionModel  0.092669  1404.666182   
5     2024-06-14      model7               XGBModel  0.091234  1333.922406   
0     2024-06-14      model1  LinearRegressionModel  0.052259   824.558770   
1     2024-06-14      model2  LinearRegressionModel  0.051878   817.262735   
2     2024-06-14      model3  LinearRegressionModel  0.077653  1139.373683   
3     2024-06-14      model4  LinearRegressionModel  0.068839  1073.844087   
4     2024-06-14      model5  LinearRegressionModel  0.071172  1121.572839   
5     2024-06-14      model7               XGBModel  0.077308  1153.257120   
0     2024-06-14      model1  LinearRegressionModel  0.083502   187.714005   
1     2024-06-14      model2  LinearRegressionModel  0.083141   188.657150   
2     2024-06-14      model3  LinearRegressionModel  0.131489   266.710639   
3     2024-06-14      model4  LinearRegressionModel  0.125430   257.151173   
4     2024-06-14      model5  LinearRegressionModel  0.123442   251.715517   
5     2024-06-14      model7               XGBModel  0.129947   261.224281   
0     2024-06-14      model1  LinearRegressionModel  0.100615    18.513913   
1     2024-06-14      model2  LinearRegressionModel  0.100654    18.651437   
2     2024-06-14      model3  LinearRegressionModel  0.150168    30.266325   
3     2024-06-14      model4  LinearRegressionModel  0.127715    25.427010   
4     2024-06-14      model5  LinearRegressionModel  0.129275    25.454507   
5     2024-06-14      model7               XGBModel  0.114773    22.695727   

   coverage                                           comments subba  
0  0.725000  LM model with lags, training with 2 years of h...  PGAE  
1  0.729167  LM model with lags, training with 3 years of h...  PGAE  
2  0.537500                                 Model 2 with lag 1  PGAE  
3  0.725000                       Model 1 with additional lags  PGAE  
4  0.725000                       Model 1 with additional lags  PGAE  
5  0.670833                                  XGBoost with lags  PGAE  
0  0.929167  LM model with lags, training with 2 years of h...   SCE  
1  0.929167  LM model with lags, training with 3 years of h...   SCE  
2  0.695833                                 Model 2 with lag 1   SCE  
3  0.887500                       Model 1 with additional lags   SCE  
4  0.862500                       Model 1 with additional lags   SCE  
5  0.775000                                  XGBoost with lags   SCE  
0  0.933333  LM model with lags, training with 2 years of h...  SDGE  
1  0.933333  LM model with lags, training with 3 years of h...  SDGE  
2  0.770833                                 Model 2 with lag 1  SDGE  
3  0.879167                       Model 1 with additional lags  SDGE  
4  0.887500                       Model 1 with additional lags  SDGE  
5  0.875000                                  XGBoost with lags  SDGE  
0  0.887500  LM model with lags, training with 2 years of h...   VEA  
1  0.833333  LM model with lags, training with 3 years of h...   VEA  
2  0.787500                                 Model 2 with lag 1   VEA  
3  0.850000                       Model 1 with additional lags   VEA  
4  0.850000                       Model 1 with additional lags   VEA  
5  0.858333                                  XGBoost with lags   VEA  

PGAE

Code
 eia_backtesting.plot_score(score = backtesting_output["PGAE"].leaderboard, type = "box")
Code
eia_backtesting.plot_score(score = backtesting_output["PGAE"].score, type = "line")

SCE

Code
 eia_backtesting.plot_score(score = backtesting_output["SCE"].leaderboard, type = "box")
Code
eia_backtesting.plot_score(score = backtesting_output["SCE"].score, type = "line")

SDGE

Code
 eia_backtesting.plot_score(score = backtesting_output["SDGE"].leaderboard, type = "box")
Code
eia_backtesting.plot_score(score = backtesting_output["SDGE"].score, type = "line")

VEA

Code
 eia_backtesting.plot_score(score = backtesting_output["VEA"].leaderboard, type = "box")
Code
eia_backtesting.plot_score(score = backtesting_output["VEA"].score, type = "line")